%ADASS_PROCEEDINGS_FORM%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
%
% TEMPLATE.TEX -- ADASS Conference Proceedings template.
%
% Use this template to create your proceedings paper in LaTeX format
% by following the instructions given below.  Much of the input will
% be enclosed by braces (i.e., { }).  The percent sign, "%", denotes
% the start of a comment; text after it will be ignored by LaTeX.  
% You might also notice in some of the examples below the use of "\ "
% after a period; this prevents LaTeX from interpreting the period as
% the end of a sentence and putting extra space after it.  
% 
% You should check your paper by processing it with LaTeX.  For
% details about how to run LaTeX as well as how to print out the User
% Guide, consult the README file.  You should also consult the sample
% LaTeX papers, sample1.tex and sample2.tex, for examples of including
% figures, html links, special symbols, and other advanced features.
%
%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%%
% Note that the primary style file is that from the ASP Conf. Series; ADASS style 
% elements are included by an additional \usepackage. You may use other 
% _standard_ packages if needed, such as lscape, psfig, epsf, and graphicx, 
% although these packages may already be installed on your system. 
%
\documentclass[11pt,twoside]{article}  % Leave intact
\usepackage{asp2006}
\usepackage{adassconf}

% Set counters for footnotes and sectioning, which is needed when 
% constructing the full volume of all papers. 
% DO NOT DELETE. 
\setcounter{equation}{0}
\setcounter{figure}{0}
\setcounter{footnote}{0}
\setcounter{section}{0}
\setcounter{table}{0}

\begin{document}   % Leave intact

%-----------------------------------------------------------------------
%			    Paper ID Code
%-----------------------------------------------------------------------
% Enter the proper paper identification code.  The ID code for your paper 
% is the session number associated with your presentation as published 
% in the official conference proceedings.  You can find this number by 
% locating your abstract in the printed proceedings that you received 
% at the meeting, or on-line at the conference web site.
%
% This identifier will not appear in your paper; however, it allows different
% papers in the proceedings to cross-reference each other.  Note that
% you should only have one \paperID, and it should not include a
% trailing period.
%
% EXAMPLE: \paperID{O4.1}
% EXAMPLE: \paperID{P2.7}

\paperID{P12}

%-----------------------------------------------------------------------
%		            Paper Title 
%-----------------------------------------------------------------------
% Enter the title of the paper.
%
% EXAMPLE: \title{A Breakthrough in Astronomical Software Development}

\title{A meta-data layer for astronomical archives}

%-----------------------------------------------------------------------
%          Short Title & Author list for page headers
%-----------------------------------------------------------------------
% Please supply the author list and the title (abbreviated if necessary) as 
% arguments to \markboth.
%
% The author last names for the page header must appear in one of 
% these formats:
%
% EXAMPLES:
%     LASTNAME
%     LASTNAME1 and LASTNAME2
%     LASTNAME1, LASTNAME2, and LASTNAME3
%     LASTNAME et al.
%
% Use the "et al." form in the case of four or more authors.
%
% If the title is too long to fit in the header, shorten it: 
%
% EXAMPLE: change
%    Rapid Development for Distributed Computing, with Implications for the Virtual Observatory
% to:
%    Rapid Development for Distributed Computing

\markboth{Ch\'ereau, Rino, and Marcos}{A meta-data layer for astronomical archives}

%-----------------------------------------------------------------------
%		          Authors of Paper
%-----------------------------------------------------------------------
% Enter the authors followed by their affiliations.  The \author and
% \affil commands may appear multiple times as necessary.  List each
% author by giving the first name or initials first followed by the
% last name. Do not include street addresses and postal codes, but 
% do include the country name or abbreviation. 
%
% If the list of authors is lengthy and there are several institutional 
% affiliations, you can save space by using the \altaffilmark and \altaffiltext 
% commands in place of the \affil command.
%
% EXAMPLE: 
%      \author{Raymond Plante, Doug Roberts, 
%                  R.\ M.\ Crutcher\altaffilmark{1}}
%      \affil{National Center for Supercomputing Applications, 
%                 University of Illinois, Urbana, IL, USA}
%      \author{Tom Troland}
%      \affil{University of Kentucky, Lexington, KY, USA}
%
%      \altaffiltext{1}{Astronomy Department, UIUC}
%
% In this example, the first three authors, "Plante", "Roberts", and
% "Crutcher" are affiliated with "NCSA".  "Crutcher" has an alternate 
% affiliation with the "Astronomy Department".  The fourth author,
% "Troland", is affiliated with "University of Kentucky"

\author{Fabien Ch\'ereau, Bruno Rino, Diego Marcos}
\affil{Virtual Observatory Project Office,  ESO, Germany}

%-----------------------------------------------------------------------
%			 Contact Information
%-----------------------------------------------------------------------
% This information will not appear in the paper but will be used by
% the editors in case you need to be contacted concerning your
% submission.  Enter your name as the contact along with your email
% address.
% 
% EXAMPLE:  \contact{Dennis Crabtree}
%           \email{crabtree@cfht.hawaii.edu}

\contact{Fabien Ch\'ereau}
\email{fchereau@eso.org}

%-----------------------------------------------------------------------
%		      Author Index Specification
%-----------------------------------------------------------------------
% Specify how each author name should appear in the author index.  The 
% \paindex{ } should be used to indicate the primary author, and the
% \aindex for all other co-authors.  You MUST use the following
% syntax: 
%
% SYNTAX:  \aindex{Lastname, F.~M.}
% 
% where F is the first initial and M is the second initial (if used). Please 
% ensure that there are no extraneous spaces anywhere within the command 
% argument. This guarantees that authors that appear in multiple papers
% will appear only once in the author index. Authors must be listed in the order
% of the \paindex and \aindex commmands.
%
% EXAMPLE: \paindex{Crabtree, D.}
%          \aindex{Manset, N.}        
%          \aindex{Veillet, C.}        

\paindex{Ch\'ereau, F.}
\aindex{Rino, B.}
\aindex{Marcos, D.}

%-----------------------------------------------------------------------
%			Subject Index keywords
%-----------------------------------------------------------------------
% Enter up to 6 keywords that are relevant to the topic of your paper.  These 
% will NOT be printed as part of your paper; however, they will guide the creation 
% of the subject index for the proceedings.  Please use entries from the
% standard list where possible, which can be found in the index for the 
% ADASS XVI proceedings. Separate topics from sub-topics with an exclamation 
% point (!). 
%
% EXAMPLE:  \keywords{astronomy!radio, computing!grid, data management!workflows, 
%     instrumentation!control}

\keywords{archives!services, data!metadata, databases!design, data management!data access, software systems!design, standards!data}

%-----------------------------------------------------------------------
%			       Abstract
%-----------------------------------------------------------------------
% Type abstract in the space below.  Consult the User Guide and Latex
% Information file for a list of supported macros (e.g. for typesetting 
% special symbols). Do not leave a blank line between \begin{abstract} 
% and the start of your text.

\begin{abstract}          % Leave intact
In many large observatories, meta-data is spread across a variety of sources (databases, text files, personal knowledge), has varying degrees of quality (in terms of completeness, correctness and precision), and operational constraints abound (one cannot risk locking the database with large queries, incorrect meta-data cannot always be corrected). Hence we propose the concept of a ``meta-data'' layer service which will shield the query services from the complexity of data sources. Based on a simple but carefully defined standard, this service aims to be generic enough to be adopted across the Virtual Observatory (VO) community, allowing interoperability of the archives at a level not yet possible in the VO.
\end{abstract}

%-----------------------------------------------------------------------
%			      Main Body
%-----------------------------------------------------------------------
% Place the text for the main body of the paper here.  You should use
% the \section command to label the various sections; use of
% \subsection is optional.  Significant words in section titles should
% be capitalized.  Sections and subsections will be numbered
% automatically. 
%
% EXAMPLE:  \section{Introduction}
%           ...
%           \subsection{Our View of the World}
%           ...
%           \section{A New Approach}
%
% It is recommended that you look at the sample paper sample2.tex
% for examples of formatting references, footnotes, figures, equations, 
% html links, lists, and other features.  

\section{The problem}
As astronomical archives are getting larger, so is the demand for smarter and richer services from the community. That is how the Virtual Observatory (VO) came about: to enable scientists to find and interpret the data they want, without knowledge of how it was obtained, without requiring them to read through the instrument documentation to understand archival data (Hanisch 2004). \\

Meta-data is at the heart of the VO (Plante 2004). Correct and precise meta-data enable the value of query services, and VO services in particular. However, the design of many large astronomical archives is still primarily focused on supporting acquisition and storage of data. This results in meta-data being spread across a complex mesh of sources (databases, text files, personal knowledge), with varying degrees of quality (in terms of completeness, correctness and precision), and with many operational constraints (one cannot risk locking the database with large queries, incorrect meta-data cannot always be corrected).\smallskip  \\* 
This complexity of the data sources alone makes building innovative services a daunting task. And as of 2009 the creation of true cross-archive science query services still remains an unreachable dream.\smallskip  \\* 
This leads to the following two conclusions: quality assurance should be a reusable effort, from which all (current and future) query services benefit; operational constraints suggests that queries services should be based on separate systems than the observatory operations.

\section{Proposed solution}
The proposed solution is built around a meta-data layer (Figure \ref{fig:1}) that shields the query services from the complexity of the data sources, and shields the data sources from the demands of the query services.

\begin{figure}[!ht]
\epsscale{0.70}
\plotone{P12.chereau.fig1.eps}
\caption{Concept of the meta-data layer.} \label{fig:1}
\end{figure}

The roles in the data flow are clearly identified and decoupled:
\begin{itemize}
 \item Ingestion scripts fill the meta-data layer from the operational databases.
 \item Crawlers update service-specific databases taking information only from the meta-data layer changed since the last crawl.
 \item Clients perform queries on the service-specific databases.
\end{itemize}
\smallskip
Query services never hit the operational databases; only the ingestion scripts do. This single access point makes it feasible to define rates of ingestion that do not impact operations.\smallskip  \\* 
Part of the ingestion scripts correct and add information not found in operational databases. Concentrating these tasks makes inconsistencies between the operational databases and the meta-data layer clear and manageable. \smallskip  \\* 
The data model of the meta-data layer should be flexible. It must cope easily with the rapidly evolving requirements of existing and emerging query services.\smallskip  \\* 
Historical changes in the meta-data itself must be recorded. While operational databases might only keep the latest version of metadata, for query services it can be important to easily return to earlier versions.


\section{Service interface}
The meta-data layer service's application programming interface (API) is kept deliberately simple: it allows only to get one meta-data file encoded as JSON (Crockford 2006) and to list the content of the layer (with support for versioning). These two features are the strict minimum needed for a crawler to go through the whole content of the exposed meta-data. It is important to notice that there is no query capability at this level.

\subsection{REST API}
The API defines the following methods, following the REST architectural style (Fielding 2002):

\begin{table}
\begin{center}\scriptsize
\begin{tabular}{ l p{10cm} }
\hline
/ 
& 
Return the list of ids of all documents of the meta-data layer
\\ \hline
/?op=size 
& 
Return the number of elements of the meta-data layer
\\ \hline
/?changedsince={ts} 
& 
Return the list of ids of documents which were modified since date ts
\\ \hline
/id 
& 
Return the document for this id or an HTTP 404 error if there is no
document for the id
\\ \hline
/id?v={ts} 
& 
Return the document for this id at version v or HTTP 404 error if there
is no document for the id/version  e.g. /id?v=2009-06-16T09:07:05
\\ \hline
/id?op=timestamp 
& 
Return the last version number of the document or an HTTP 404 error
if there is no document for this id
\\ \hline
/id?op=history 
& 
Return the list of previous versions numbers of the document or an
HTTP 404 if there is no document for this id
\\ \hline
\end{tabular}
\end{center}
\end{table}

\subsection{JSON : Query result format}
The content of the meta-data files are returned as JSON files. The files are semi-structured, based on schemas carefully defined for common types of documents (images, spectra, etc...) but also allowing the flexibility to add custom items specific to a subset of documents, e.g. for storing information items specific to a given telescope or institution

\begin{verbatim}
"id": "LP175C0685.WFI.2005-12-09T05:12:15.441",
"type": "image",
"title": "WFI.2005-12-09T05:12:15.441.fit",
"publisher": "ESO SAF",
"collection": "175.C-0685",
"license": "ESO Data License",
"creator": "Dupondt",
"characterization": {
  "spatialAxis": {
    "footprint": {
      "worldCoords": [[[122.17098758933295, -48.631020973582558]..
         [123.02688788655796, -49.176814585277484], [122.1709875..
      },
    "centralPos": [122.59955700853416, -48.90469867250895]
  },
  "temporalAxis": {
    "boundingBox": [53713.216845379997, 53713.218233320003], 
  ...
\end{verbatim}

\section{A multi-archive use case}
The meta-data layer allows to completely decouple the roles of the meta-data providers and the one of the services providers (query service, display service etc..). This unleashes the possibility of building cross-archives services using exactly the same principles (API+crawler) as for the single archive use case, provided that the various data centres adopt the same conventions for the service interface API.

\begin{figure}[!ht]
\epsscale{0.70}
\plotone{P12.chereau.fig2.eps}
\caption{Multi-archive use case.} \label{fig:2}
\end{figure}

Figure \ref{fig:2} is an example of a cross-archive search engine based on the content from three data centres X, Y and Z. A single crawler is used for regularly harvesting each of the three meta-data layers. From each of them, it extracts the information the it needs to fill its internal index.

The same architecture can be used to create any number of specialized cross-archives services such as a SED (Spectral Energy Distribution) builder, or a catalog cross matching service.

The design and adoption of such a simple but powerful API might well be the currently missing corner stone on which to build the rest of the VO.


%-----------------------------------------------------------------------
%			      References
%-----------------------------------------------------------------------
% List your references below within the reference environment
% (i.e. between the \begin{references} and \end{references} tags).
% Each new reference should begin with a \reference command which sets
% up the proper indentation.  
%    NOTE: all citations in the text _must_ have a corresponding entry in 
%    the reference list, and all references must be cited in the text.
%
% Observe the following order when listing bibliographical 
% information for each reference:  author name(s), publication 
% year, journal name, volume, and page number for articles. 
% URLs to the reference may be given either in-line, or as a footnote. 
% Note that many journal names are available as macros; see
% the User Guide for a listing "macro-ized" journals. 
%
% EXAMPLES:  
% Reference to a Journal article:
%     \reference Cornwell, T.\ J.\ 1988, \aap, 202, 316
%
% Journal paper with more than 7 authors;
%     \reference Hanisch, R.\ et al.\ 2001, \aap, 376, 359
%
% Reference to an SPIE paper:
%     \reference Noordam, J.~E.\ 2004, Proc.\ SPIE, 5489, 817
%
% Reference to a contribution to a proceedings (not ADASS)
%     \reference Schmitz, M., Helou, G., Dubois, P., LaGue, C., Madore,B., Corwin, H.~G., Jr., 
%          \& Lesteven, S.\ 1995, in Information \& On-Line Data in Astronomy, 
%          ed.\ D.\ Egret \& M.~A.\ Albrecht (Dordrecht: Kluwer Academic Publishers), 259
%
% Reference to a paper in an earlier ADASS proceedings:
%     \reference Kantor, J., et al.\ 2007, \adassvii, 3
%
% Reference to a paper in the current ADASS:
%     \reference Hanisch, R.~J.\ 2008, \adassxvii, \paperref{O1.3}
% 
% Reference to a book:
%     \reference Jacobson, I.\ Booch, G., \& Rumbaugh, J.\ 1999, 
%            The Unified Software Development Process (Reading, MA: Addison-Wesley)
%
% Reference to a thesis:
%     \reference Gering, D.\ 1999, Master's Thesis, Massachusetts Institute of Technology
% 
% Reference to a purely on-line resource:
%     \reference Staveley-Smith, L.\ 2006, ATNF SKA Memo~6, http://www.atnf.csiro.au/ska
%
% Note the following tricks used in the example above:
%
%   o  \& is used to format an ampersand symbol (&).
%   o  \'e puts an accent agu over the letter e.  See the User Guide
%      and the sample files for details on formatting special
%      characters.  
%   o  "\ " after a period prevents LaTeX from interpreting the period 
%      as an end of a sentence.
%   o  \aj is a macro that expands to "Astron. J."  See the User Guide
%      for a full list of journal macros
%   o  \adassvii is a macro that expands to the full title, editor,
%      and publishing information for the ADASS VII conference
%      proceedings.  Such macros are defined for ADASS conferences I
%      through XVI.
%   o  When referencing a paper in the current volume, use the
%      \adassxvii and \paperref macros.  The argument to \paperref is
%      the paper ID code for the paper you are referencing.  See the 
%      note in the "Paper ID Code" section above for details on how to 
%      determine the paper ID code for the paper you reference.  
%
\begin{references}
\reference Crockford, D.\  2006, 
    "The application/json Media Type for JavaScript Object Notation (JSON)"
    http://tools.ietf.org/html/rfc4627
\reference Fielding, Roy T.; Taylor, Richard N.\  2002, 
    "Principled Design of the Modern Web Architecture", 
    in ACM Transactions on Internet Technology, Vol.\  2, No.\  2, 115
\reference Plante, R.\  2004, "Scalable Metadata Deﬁnition Frameworks", 
    in Toward an International Virtual Observatory
    (Springer Berlin / Heidelberg), 106
\reference Quinn, P., Lawrence, A., Hanisch, R.\  2004, 
    "The Management, Storage, and Utilization of Astronomical Data in the 21st 
    Century", http://www.ivoa.net/Documents/lat- est/OECDWhitePaper.html
\end{references}

% Do not place any material after the references section

\end{document}  % Leave intact
